
import re
import string
import logging
import urllib2
from datetime import timedelta, datetime, date, time
from google.appengine.ext import db
from google.appengine.api import xmpp
from google.appengine.ext import webapp
from google.appengine.api import urlfetch
from google.appengine.ext.webapp.util import run_wsgi_app

####################################################################################
#
#     D a t a b a s e
#
####################################################################################

class Quakes(db.Model):
  Added     = db.DateTimeProperty(auto_now_add=True) 
  Modified  = db.DateTimeProperty(auto_now=True, auto_now_add=True) 
  Removed   = db.IntegerProperty()  
  Url       = db.StringProperty()  
  Src       = db.StringProperty()
  Eqid      = db.StringProperty()
  Version   = db.StringProperty()
  Datetime  = db.DateTimeProperty()
  Lat       = db.StringProperty()
  Lon       = db.StringProperty()
  Magnitude = db.StringProperty()
  Depth     = db.StringProperty()
  NST       = db.StringProperty()
  Region    = db.StringProperty()
  Keywords  = db.StringListProperty()
  Tags      = db.StringListProperty()


#===================================================================================
class MainHandler(webapp.RequestHandler):
  #=================================================================================
  def get(self):
    #-------------------------------------------------------------------------------
    self.redirect("/public/earthquakes.htm")
    #-------------------------------------------------------------------------------
    pass

#===================================================================================
class TagHandler(webapp.RequestHandler):
  #=================================================================================
  def get(self):
    #-------------------------------------------------------------------------------
    tag      = self.request.get('tag',       'Chile')
    count    = int(self.request.get('count', '-1'))
    callback = self.request.get('callback',   '')
    web      = int(self.request.get('web',    '0'))
    #-------------------------------------------------------------------------------
    query    = ''
    if count == -1:
      query = 'SELECT * FROM Quakes WHERE Tags = :1 ORDER BY Datetime DESC'
    else:
      query = 'SELECT * FROM Quakes WHERE Tags = :1 ORDER BY Datetime DESC LIMIT ' + str(count)
    #-------------------------------------------------------------------------------
    if callback != '':
      self.response.out.write(callback + '(');
    #-------------------------------------------------------------------------------
    self.response.out.write('[');
    if web == 1:
      self.response.out.write('<br>');
    #-------------------------------------------------------------------------------
    _events = db.GqlQuery(query, tag)
    for _event in _events:
      status = ''
      if _event.Removed == 1:
        status = '-'
      else:
        status = '+'
      self.response.out.write(
        "{'rem':'" + status + "'," +
        "'url':'" + _event.Url + "'," +
        "'utc':'" + str(_event.Datetime) + "'," +
        "'lat':'" + _event.Lat + "'," +
        "'lon':'" + _event.Lon + "'," +
        "'mag':'" + _event.Magnitude + "'," +
        "'dep':'" + _event.Depth + "'," +
        "'nst':'" + _event.NST + "'," +
        "'reg':'" + _event.Region.replace('\'', '\\\'') + "'},"
      )
      '''
      self.response.out.write(
        "{'rem':'" + status + "'," +
        "'add':'" + str(_event.Added) + "'," +
        "'mod':'" + str(_event.Modified) + "'," +
        "'url':'" + _event.Url + "'," +
      '''
      if web == 1:
        self.response.out.write('<br>');
    #-------------------------------------------------------------------------------
    self.response.out.write(']');
    if callback != '':
      self.response.out.write(');');
    #-------------------------------------------------------------------------------
    pass

#===================================================================================
class FetchHandler(webapp.RequestHandler):
  #=================================================================================
  def get(self):
    #-------------------------------------------------------------------------------
    hours    = int(self.request.get('hours',    '1'))
    callback = self.request.get('callback', '')
    web      = int(self.request.get('web', '0'))
    #-------------------------------------------------------------------------------
    interval = datetime.now() - timedelta(hours=hours)
    time_query = interval.strftime("%Y, %m, %d, %H, %M")
    #-------------------------------------------------------------------------------
    if callback != '':
      self.response.out.write(callback + '(');
    #-------------------------------------------------------------------------------
    self.response.out.write("{'i':60,'d':[");
    if web == 1:
      self.response.out.write('<br>');
    #-------------------------------------------------------------------------------
    _events = db.GqlQuery('SELECT * FROM Quakes WHERE Datetime > DATETIME('+time_query+', 00) ORDER BY Datetime DESC')
    for _event in _events:
      status = ''
      if _event.Removed == 1:
        status = '-'
      else:
        status = '+'
      self.response.out.write(
        "{'rem':'" + status + "'," +
        "'url':'" + _event.Url + "'," +
        "'utc':'" + str(_event.Datetime) + "'," +
        "'lat':'" + _event.Lat + "'," +
        "'lon':'" + _event.Lon + "'," +
        "'mag':'" + _event.Magnitude + "'," +
        "'dep':'" + _event.Depth + "'," +
        "'nst':'" + _event.NST + "'," +
        "'reg':'" + _event.Region.replace('\'', '\\\'') + "'},"
      )
      '''
      self.response.out.write(
        "{'rem':'" + status + "'," +
        "'add':'" + str(_event.Added) + "'," +
        "'mod':'" + str(_event.Modified) + "'," +
        "'url':'" + _event.Url + "'," +
      '''
      if web == 1:
        self.response.out.write('<br>');
    #-------------------------------------------------------------------------------
    self.response.out.write(']}');
    if callback != '':
      self.response.out.write(');');
    #-------------------------------------------------------------------------------
    pass

#===================================================================================
class CronHandler(webapp.RequestHandler):
  #=================================================================================
  def get(self):
    
    min = -1
    max = -1
    
    inserted = 0
    inserted_earthquakes = []
    
    existed  = 0
    
    received_urls  = []
    received_dates = []
    
    count  = int(self.request.get('count',  '-1'))
    days   = int(self.request.get('days',    '1'))
    #delete = int(self.request.get('delete',  '0'))
    #offset = int(self.request.get('offset', '-1')) disable this, it does harm
    delete = 0
    offset = -1
    
    #if delete == 1:
    #  q = db.GqlQuery("SELECT * FROM Quakes")
    #  results = q.fetch(500)
    #  db.delete(results)
    #  return
    
    #---------------------------------------------------------------------------------
    result = urlfetch.fetch(url='http://earthquake.usgs.gov/earthquakes/catalogs/eqs' + str(days) + 'day-M1.txt')
    if (result.status_code == 200):
      lines = result.content.split('\n')
      lines.pop(0)
      
      if (offset != -1):
        min = offset
        if (count != -1):
          max = min + count
          lines = lines[min:max]
        else:
          lines = lines[min:]
      else:
        if (count != -1):
          lines = lines[:count]
        
      
      self.response.out.write("""
        <html>
          <body>
      """)
      
      if (offset != -1):
        if (count != -1):
          self.response.out.write(str(min) + " -> " + str(max) + "<br>")
        else:
          self.response.out.write(str(min) + " -> remaining<br>")
      else:
        if (count != -1):
          self.response.out.write("0 -> " + str(max) + "<br>")
        else:
          self.response.out.write("0 -> remaining<br>")
      
      p1 = re.compile(r',"|",|"')
      for line in lines:
        if line <> "":
          
          # coarsely split the line
          segments    = p1.split(line)
          
          # get the catalog info first, since it contains all the data needed to see
          # if this event is already in the datastore
          catalog     = segments[0].split(',')
          
          # we'll be using this a couple of times
          url = catalog[0] + catalog[1]
          dateandtime = datetime.strptime(segments[1], "%A, %B %d, %Y %H:%M:%S UTC")
          
          # keep in mind that we got provided this url (we're need to check for removals after inserting)
          received_urls.append(url)
          received_dates.append(dateandtime)
          
          _events = db.GqlQuery("SELECT * FROM Quakes WHERE Url='" + url + "'")
          _event = _events.get()
          if _event == None:
            
            # this event is not in the datastore, so we continue processing the line
            geological  = segments[2].split(',')
            region      = segments[3]
            
            keywords    = region.split(', ')
            tags = []
            for keyword in keywords:
              tags.extend(keyword.split(' '))
            
            # and we insert the gathered information into the datastore
            new_event           = Quakes()
            new_event.Url       = url
            new_event.Src       = catalog[0]
            new_event.Eqid      = catalog[1]
            new_event.Version   = catalog[2]
            new_event.Datetime  = dateandtime
            new_event.Lat       = geological[0]
            new_event.Lon       = geological[1]
            new_event.Magnitude = geological[2]
            new_event.Depth     = geological[3]
            new_event.NST       = geological[4]
            new_event.Region    = region
            new_event.Keywords  = keywords
            new_event.Tags      = tags
            new_event.put()
            
            inserted = inserted + 1
            inserted_earthquakes.append([str(geological[2]), region])

            # and print out the data
            self.response.out.write("inserted: " + str(catalog) + " - " + str(dateandtime) + " - " + str(geological) + " - " + region  + "<br>")
          else:
            # if a removed item reappears in the list, remove the flag
            if _event.Removed == 1:
              _event.Removed = 0
              _event.put()
            existed = existed + 1
            #self.response.out.write(url + " exists<br>")
      
      # at this point we have inserted all the new earthquakes
      
      last_url  = received_urls[len(received_urls)-1]
      last_date = received_dates[len(received_dates)-1]
      self.response.out.write("<br>oldest received url: " + last_url  + " at " + str(last_date))
      
      stored_urls = []
      
      time_query = last_date.strftime("%Y, %m, %d, %H, %M, %S")
      _events = db.GqlQuery('SELECT * FROM Quakes WHERE Datetime >= DATETIME('+time_query+') ORDER BY Datetime DESC')
      for _event in _events:
        stored_urls.append(_event.Url)
      
      missing_urls = set(stored_urls).difference(received_urls)

      self.response.out.write("<br>total of " + str(len(received_urls)) + " items received, total of " + str(len(stored_urls)) + " items in storage,  missing urls " + str(len(missing_urls)) + "<br>")
      
      for missing_url in missing_urls:
        _events = db.GqlQuery('SELECT * FROM Quakes WHERE Url = :1', missing_url)
        for _event in _events:
          _event.Removed = 1
          _event.put()
          self.response.out.write(_event.Url + " - " + str(_event.Datetime) + " - " + _event.Region + "<br>")
      
      self.response.out.write("""
          </body>
        </html>
      """)
      if (inserted > 0):
        new_earthquakes = ''
        for earthquake in inserted_earthquakes:
          new_earthquakes = new_earthquakes + earthquake[0] + ' - ' + earthquake[1] + '\n'
        #xmpp.send_message('...@gmail.com', new_earthquakes)
      pass
    else:
      #xmpp.send_message('...@gmail.com', "There was a problem. Code:" + str(result.status_code))
      pass
    #-------------------------------------------------------------------------------
    pass

####################################################################################
#
#     a p p l i c a t i o n
#
####################################################################################

application = webapp.WSGIApplication(
  [
    ('/',            MainHandler),
    ('/web/fetch',   FetchHandler),
    ('/web/tag',     TagHandler),
    ('/cron/update', CronHandler)
  ],
  debug=True
)

def main():
  run_wsgi_app(application)

if __name__ == "__main__":
  main()
